#!/bin/bash
#Copyright 2005-2017, Lime Technology
#License: GPLv2 only

# This is the 'mover tuning' script used for moving files between the cache disk and main array.
# It is typically invoked via cron.

# First we check if it's valid for this script run: there must be a cache disk present and
# an instance of the script must not already be running.

# Next, check each of the top-level directories (shares) on the cache disk.
# For all share with 'Use Cache' setting set to "prefer" or "yes", we use 'find' to
# create a filtered file list of that share directory.
# For all share with 'Use Cache' setting set to "only", we use 'du' or 'zfs list' to
# get total size of that share directory.

# The list is sorted by "Use cache", increasing age, pool, and file inode, giving priority for being 
# on cache to "cache only" shares, then "cache prefer" by moving newest from array to cache and older to array, 
# and finally to "cache yes" share by moving only from cache to array.
# Please note that if age setting is set to something else than "Auto (smart cache)" this script is actually
# dumb and do not check for size and free space and rely on your own calculations.
# Files at the top level of the cache or an array disk (i.e not in a share) are never moved.

# The list is then passed to original unraid mover.
# For each file, if the file is not "in use" by any process (as detected by 'fuser' command),
# then the file is moved, and upon success, deleted from the source disk.  If the file already
# exists on the target, it is not moved and the sourceis not deleted.  All meta-data of moved
# files/directories is preserved: permissions, ownership, extended attributes, and access/modified
# timestamps.

# If an error occurs in copying a file, the partial file, if present, is deleted and the
# operation continues on to the next file.

PIDFILE="/var/run/mover.pid"
SOFTSTOPFILE="/var/run/moversoft.stop"
SOFTSTOP_ACK=false
UNRAIDCFGFILE="/boot/config/share.cfg"
MOVERTUNINGCFGFILE="/boot/config/plugins/ca.mover.tuning/ca.mover.tuning.cfg"
LOGLEVEL=1
TOTALPRIMARYFILES=0
REMAININGPRIMARYFILES=0
TOTALPRIMARYSIZE=0
REMAININGPRIMARYSIZE=0
TOTALSECONDARYFILES=0
REMAININGSECONDARYFILES=0
TOTALSECONDARYSIZE=0
REMAININGSECONDARYSIZE=0
TOTALUNATTENDEDFILES=0
TOTALUNATTENDEDSIZE=0
REMAININGUNATTENDEDFILES=0
REMAININGUNATTENDEDSIZE=0
overrideFlag=0

NOW=$(date +"%FT%H%M%S")
FILTERED_FILELIST="/tmp/ca.mover.tuning/Filtered_files_$NOW.list"
MOVER_ACTIONLIST="/tmp/ca.mover.tuning/Mover_action_$NOW.list"
STATS_FILE="/tmp/ca.mover.tuning/Summary_$NOW.txt"
MOVER_STATUS="/usr/local/emhttp/state/mover.ini"
MOVER_LOG="/tmp/ca.mover.tuning/Mover_tuning_$NOW.log"
GUI_CURRENT_FILE=""
GUI_ACTION="Calculating"

function is_zfs_mountpoint_fstype() {
    local result
    result=$(findmnt $1 -o fstype | tail -n 1)
    [ "$result" = "zfs" ]
    return
}

function titleLine() {
  local title="$1"
  local padding_char="$2"
  local line_length=71
  local padding_left=$((line_length - ${#title} / 2))
  local padding_right=$(( padding_left - ${#title} % 2 -1 ))
  local left_padding=$(printf "%*s" "$padding_left" | tr ' ' "$padding_char")
  local right_padding=$(printf "%*s" "$padding_right" | tr ' ' "$padding_char")
  echo "${left_padding}$padding_char $title $padding_char${right_padding}"
}

# Function to convert byte size to human-readable format
function bytes_to_human() {
  echo "$(numfmt $1 --to=iec-i)B"
}


function human_to_bytes() {
  echo $(numfmt $1 --from=iec)
}

# Function for verbose output and logging
mvlogger() {

    if [ ! -d "/tmp/ca.mover.tuning" ]; then
        echo "/tmp/ca.mover.tuning Directory does not exist. Creating it"
        mkdir -p /tmp/ca.mover.tuning
    fi

    if [ $LOGLEVEL = 1 ]; then
        if [ -t 1 ]; then # running from terminal, echoing date
            echo "$(date +"%T.%3N") $1"
        else # running from cron
            echo "$1"
        fi
    fi
    echo "$(date +"%T.%3N") $1" >>$MOVER_LOG
}

#Moved variable assignment into a function
getMoverSettings() {
    #Use the config file instead of input variables (ease execution from command line)
    local config_file="$1"
    cfg() {
        local param="$1"
        local value
        while IFS='=' read -r key val; do
        [ "$key" == "$param" ] && { value="$val"; break; }
        done < "$config_file"
        echo "${value//\"/}"  # Print value or empty string if not found, without quotes
    }

    timezone=$(grep "timeZone=" "/boot/config/ident.cfg" | cut -d '"' -f 2)

    if [[ ! -f "$config_file" ]]; then
        mvlogger "Error: Config file '$config_file' not found."
        return 1
    fi

    ## Read in default threshold limit.
    ## [[ ! "$config_file" =~ shareOverrideConfig ]] ensure that we are not reading share custom settings
    
    if [[ ! "$config_file" =~ shareOverrideConfig ]] ;then
        MOVINGPCTTHRESHOLD=$(cfg movingThreshold)
        [ -z $MOVINGPCTTHRESHOLD ] && MOVINGPCTTHRESHOLD=0
        mvlogger "Using global moving threshold: $MOVINGPCTTHRESHOLD %"
    elif [ ! -z $(cfg movingThreshold) ]; then
        MOVINGPCTTHRESHOLD=$(cfg movingThreshold)
        mvlogger "Using share moving threshold: $MOVINGPCTTHRESHOLD %"
    fi

    if [[ ! "$config_file" =~ shareOverrideConfig ]] ;then
        FREEINGPCTLIMIT=$(cfg freeingThreshold)
        if [ -z $FREEINGPCTLIMIT ]; then
            FREEINGPCTLIMIT=$MOVINGPCTTHRESHOLD
            echo "freeingThreshold=\"$FREEINGPCTLIMIT\"" >> $config_file
        fi
        mvlogger "Using global freeing threshold: $FREEINGPCTLIMIT %"
    elif [ ! -z $(cfg freeingThreshold) ]; then
        FREEINGPCTLIMIT=$(cfg freeingThreshold) 
        mvlogger "Using share freeing threshold: $FREEINGPCTLIMIT %"
    fi

    # Read other thresholds
    if [ -z $(cfg omovercfg) ]; then
        if [[ ! "$config_file" =~ shareOverrideConfig ]] ;then
            mvlogger "No Move All from Primary->Secondary shares argument provided, defaulting to no"
            echo 'omovercfg="no"' >> $config_file
            OMOVERTHRESH=""
        fi
    else
        if [ $(cfg omovercfg) = "yes" ]; then
            OMOVERTHRESH=$(cfg omoverthresh)
            [ ! -z $OMOVERTHRESH ] && mvlogger "Primary threshold to Move all Primary->Secondary shares to secondary: $OMOVERTHRESH %"
        else
            [[ ! "$config_file" =~ shareOverrideConfig ]] && OMOVERTHRESH=""
        fi
    fi
    
    # Read filters parameters
    if [ -z $(cfg age) ]; then
        if [[ ! "$config_file" =~ shareOverrideConfig ]] ;then
            mvlogger "No Age argument provided, defaulting to no"
            echo 'age="no"' >> $config_file
            AGE=0
        fi
    else
        if [ $(cfg age) = "yes" ]; then
            AGE=$(cfg daysold)
            if [ $AGE -eq -1 ]; then 
                mvlogger "Age: Automatic (smart caching)"
            else
                mvlogger "Age: $AGE"
            fi
        else
            AGE=0
            mvlogger "Age: $(cfg age) = $AGE ; daysold: $(cfg daysold)"
        fi
    fi

    if [ -z $(cfg afterScript) ]; then
        #mvlogger "No After Script argument provided"
        [[ ! "$config_file" =~ shareOverrideConfig ]] && AFTERSCRIPT=""
    else
        AFTERSCRIPT=$(cfg afterScript)
        mvlogger "After script: $AFTERSCRIPT"
    fi

    if [ -z $(cfg beforeScript) ]; then
        #mvlogger "No Before Script argument provided"
        [[ ! "$config_file" =~ shareOverrideConfig ]] && BEFORESCRIPT=""
    else
        BEFORESCRIPT=$(cfg beforeScript)
        mvlogger "Before script: $BEFORESCRIPT"
    fi

    if [ -z $(cfg cleanFolders) ]; then
        if [[ ! "$config_file" =~ shareOverrideConfig ]] ;then
            mvlogger "No Clean folders argument provided, defaulting to no"
            CLEANFOLDERS="no"
            echo 'cleanFolders="no"' >> $config_file
        fi
    else
        CLEANFOLDERS=$(cfg cleanFolders)
        if [ $CLEANFOLDERS = "yes" ]; then
            mvlogger "Clean Folders: $CLEANFOLDERS"
        fi
    fi

    if [ -z $(cfg ctime)  ]; then
        if [[ ! "$config_file" =~ shareOverrideConfig ]] ;then
            mvlogger "No CTIME argument provided, defaulting to no"
            CTIME="no"
            echo 'ctime="no"' >> $config_file
        fi
    else
        CTIME=$(cfg ctime)
        [ $CTIME = "yes" ] && mvlogger "CTIME: $CTIME"
    fi

    if [ -z $(cfg enableTurbo) ]; then
        if [[ ! "$config_file" =~ shareOverrideConfig ]] ;then
            mvlogger "No Turbo Mode argument provided, defaulting to no"
            ENABLETURBO="no"
            echo 'enableTurbo="no"' >> $config_file
        fi
    else
        ENABLETURBO=$(cfg enableTurbo)
        [ $ENABLETURBO = "yes" ] && mvlogger "Enable Turbo: $ENABLETURBO"
    fi

    if [ -z $(cfg filelistf) ]; then
        if [[ ! "$config_file" =~ shareOverrideConfig ]] ;then
            mvlogger "No Skipfiles argument provided, defaulting to no"
            echo 'filelistf="no"' >> $config_file
        fi
    else
        if [ $(cfg filelistf) = "yes" ]; then
            SKIPFILESLIST=$(cfg filelistv)
            mvlogger "Skip file list: $SKIPFILESLIST"
        else
            [[ ! "$config_file" =~ shareOverrideConfig ]] && SKIPFILESLIST=""
        fi
    fi

    if [ -z $(cfg filetypesf) ]; then
        if [[ ! "$config_file" =~ shareOverrideConfig ]] ;then
            mvlogger "No Skip File Types argument provided, defaulting to no"
            echo 'filetypesf="no"' >> $config_file
            SKIPFILETYPES=""            
        fi
    else
        if [ $(cfg filetypesf) = "yes" ]; then
            ###echo "Skipfiletypes supplied"
            SKIPFILETYPES=$(cfg filetypesv)
            SKIPFILETYPES=$(echo "$SKIPFILETYPES" | awk -F, '{for(i=1;i<=NF;i++) {gsub(/\.| /, "", $i); printf "%s%s", $i, (i<NF?" ":"")}}')
            mvlogger "Skip filetypes: $SKIPFILETYPES"
        else
            [[ ! "$config_file" =~ shareOverrideConfig ]] && SKIPFILETYPES=""
        fi
    fi

    if [ -z $(cfg ignoreHidden) ]; then
        if [[ ! "$config_file" =~ shareOverrideConfig ]] ;then
            mvlogger "No Ignore Hidden Files argument provided, defaulting to no"
            echo 'ignoreHidden="no"' >> $config_file
            IGNOREHIDDENFILES="no"
        fi
    else
        IGNOREHIDDENFILES=$(cfg ignoreHidden)
        [ $IGNOREHIDDENFILES = "yes" ] && mvlogger "Ignore Hidden Files: $IGNOREHIDDENFILES"
    fi
    
    if [ -z $(cfg rebalanceShares) ]; then
        if [[ ! "$config_file" =~ shareOverrideConfig ]] ;then
            mvlogger "No Rebalance argument provided, defaulting to no"
            echo 'rebalanceShares="no"' >> $config_file
            REBALANCESHARES="no"
        fi
    else
        REBALANCESHARES=$(cfg rebalanceShares)
        [ $REBALANCESHARES = "yes" ] && mvlogger "Rebalance shares: $REBALANCESHARES"
    fi

    if [ -z $(cfg sizef) ]; then
        if [[ ! "$config_file" =~ shareOverrideConfig ]] ;then
            mvlogger "No Size argument provided, defaulting to no"
            echo 'sizef="no"' >> $config_file
            SIZE=0
        fi
    else
        if [ $(cfg sizef) = "yes" ]; then
            SIZE=$(cfg sizeinM)
            [ $SIZE -gt 0 ] && mvlogger "Size supplied: $SIZE"
        else
            [[ ! "$config_file" =~ shareOverrideConfig ]] &&  SIZE=0
        fi
    fi

    if [ -z $(cfg sparsnessf) ]; then
        if [[ ! "$config_file" =~ shareOverrideConfig ]] ;then
            mvlogger "No Sparness argument provided, defaulting to no"
            echo 'sparsnessf="no"' >> $config_file
            SPARSENESS=0
        fi
    else
        if [ $(cfg sparsnessf) = "yes" ]; then
            SPARSENESS=$(cfg sparsnessv)
            [ $SPARSENESS - gt 0 ] && mvlogger "Sparness supplied: $SPARSENESS"
        else
            [[ ! "$config_file" =~ shareOverrideConfig ]] && SPARSENESS=0
        fi
    fi

    if [ -z $(cfg synchronizeCache) ]; then
        if [[ ! "$config_file" =~ shareOverrideConfig ]] ;then
            mvlogger "No Synchronize Primary to Secondary argument provided, defaulting to no"
            echo 'synchronizeCache="no"' >> $config_file
            SYNCHRONIZECACHE="no"
        fi
    else
        SYNCHRONIZECACHE=$(cfg synchronizeCache)
        if [ $SYNCHRONIZECACHE = "yes" ]; then
            LASTCACHESYNC=$(cfg lastCacheSync)
            if [[ ! "$config_file" =~ shareOverrideConfig ]]; then
                [ -z $LASTCACHESYNC ] && LASTCACHESYNC=0
                mvlogger "Synchronize Primary to Secondary: $SYNCHRONIZECACHE - Last sync date: $(TZ=$timezone date +%FT%H%M%S -d "@$LASTCACHESYNC")"
            else
                mvlogger "Synchronize Primary to Secondary: $SYNCHRONIZECACHE"
            fi
        fi
    fi

    if [ -z $(cfg resynchronizeCache) ]; then
        if [[ ! "$config_file" =~ shareOverrideConfig ]] ;then
            mvlogger "No Resynchronize argument provided, defaulting to no"
            echo 'resynchronizeCache="no"' >> $config_file
            RESYNCHRONIZECACHE="no"
        fi
    else
        RESYNCHRONIZECACHE=$(cfg resynchronizeCache)
        if [ $RESYNCHRONIZECACHE != "no" ]; then
            mvlogger "Resynchronize Primary to Secondary: $RESYNCHRONIZECACHE"
        fi
        [ $RESYNCHRONIZECACHE = "run-once" ] && RESYNCHRONIZECACHE="yes"
    fi

    if [ -z $(cfg testmode) ]; then
        if [[ ! "$config_file" =~ shareOverrideConfig ]] ;then
            mvlogger "No Test Mode argument provided, defaulting to yes"
            echo 'testmode="yes"' >> $config_file
            TESTMODE="yes"
        fi
    else
        TESTMODE=$(cfg testmode)
        [ $TESTMODE = "yes" ] && mvlogger "Test Mode: $TESTMODE"
    fi

}

resetRunOnceMoverSettings() {
    local config_file="$1"
    if grep -q "run-once" $1; then
        mvlogger "Reset 'Run-once' settings to 'no' in $1"
        sed 's|"run-once"|"no"|g' -i $1
    #else
        #mvlogger "No run-once settings to reset in $1"
    fi
}

moverStatusWrite() {
    #echo "test write"
    #Writes to file /usr/local/emhttp/state/mover.ini GUI update variables
    echo "TotalToSecondary=${TOTALPRIMARYSIZE}" >$MOVER_STATUS
    echo "RemainToSecondary=${REMAININGPRIMARYSIZE}" >>$MOVER_STATUS
    echo "TotalFromSecondary=${TOTALSECONDARYSIZE}" >>$MOVER_STATUS
    echo "RemainFromSecondary=${REMAININGSECONDARYSIZE}" >>$MOVER_STATUS
    echo "TotalFilesToSecondary=${TOTALPRIMARYFILES}" >$MOVER_STATUS
    echo "RemainFilesToSecondary=${REMAININGPRIMARYFILES}" >>$MOVER_STATUS
    echo "TotalFilesFromSecondary=${TOTALSECONDARYFILES}" >>$MOVER_STATUS
    echo "RemainFilesFromSecondary=${REMAININGSECONDARYFILES}" >>$MOVER_STATUS
    echo "File=${GUI_CURRENT_FILE}" >>$MOVER_STATUS
    echo "Action=${GUI_ACTION}" >>$MOVER_STATUS
}

createFilteredFilelist (){
    mvlogger "$(titleLine 'FILTERING FILES' '*')"

    # create a file containing the list off all shareUseCache yes or prefer whith this format:
    echo "PRIMARYSTORAGENAME|SECONDARYSTORAGENAME|SHARENAME|SHAREUSECACHE|MODIFICATIONTIME|PRIMARYSIZETRESH|FILESIZE|SPARSENESS|NBLINKS|INODE|FILEPATH" > $FILTERED_FILELIST

    for SHARECFG in /boot/config/shares/*.cfg; do
        if [ -e $SOFTSTOPFILE ]; then
            [ $SOFTSTOP_ACK = true ] || mvlogger "Soft Stopping requested" && SOFTSTOP_ACK=true
            return
        fi

        globalSkipFileTypes=$SKIPFILETYPES

        #Gathering information
        #Sharename
        SHARENAME="$(basename "$SHARECFG" .cfg)"
        # Share title line
        mvlogger "$(titleLine "Processing $SHARENAME share" '-')"

        #Check to see if this share has a mover settings override, if yes set overrideFlag and change settings
        if [ -f "/boot/config/plugins/ca.mover.tuning/shareOverrideConfig/$SHARENAME.cfg" ]; then
            SHARETUNINGFILE="/boot/config/plugins/ca.mover.tuning/shareOverrideConfig/$SHARENAME.cfg"
            if grep -qs 'moverOverride="yes"' $SHARETUNINGFILE; then
                overrideFlag=1
                mvlogger  "Settings override:"
                mvlogger  "------------------"
                getMoverSettings $SHARETUNINGFILE
                mvlogger "------------------------------------------------------------------------"
            fi
        elif [ $overrideFlag = 1 ]; then
            #else if overrideFLag set from previous loop then restore global settings
            overrideFlag=0
            #Get Mover Tuning Settings
            mvlogger "Restore global settings"
            mvlogger "-----------------------"
            getMoverSettings $MOVERTUNINGCFGFILE
            mvlogger "------------------------------------------------------------------------"
        fi

        # Determine Cache mode
        SHAREUSECACHE=$(grep shareUseCache "$SHARECFG" | cut -d'=' -f 2 | tr -d '"' | tr -d '\r')

        # Determine primary storage
        PRIMARYSTORAGENAME=$(grep "shareCachePool=" "$SHARECFG" | cut -d'=' -f 2 | tr -d '"' | tr -d '\r')
        if [ -z "$PRIMARYSTORAGENAME" ]; then
            PRIMARYSTORAGENAME="user0"
        fi
        # Check if the storage exists
        if [ ! -d "/mnt/$PRIMARYSTORAGENAME" ]; then
            # Do not process this pool if path does not exist
            mvlogger "/mnt/$PRIMARYSTORAGENAME does not exist. Is the pool still used? Consider removing $SHARECFG if not."
            mvlogger "=> Skipping"
            continue  # Move to the next iteration of the loop
        fi

        # Determine secondary storage
        SECONDARYSTORAGENAME=$(grep "shareCachePool2=" "$SHARECFG" | cut -d'=' -f 2 | tr -d '"' | tr -d '\r')
        if [ -z "$SECONDARYSTORAGENAME" ]; then
            if [ $SHAREUSECACHE = "only" ] || [ $SHAREUSECACHE = "no" ]; then
                SECONDARYSTORAGENAME="none"
            else
                SECONDARYSTORAGENAME="user0"
            fi
        fi

        #Find the current percent of used size of pool.
        if is_zfs_mountpoint_fstype "/mnt/$PRIMARYSTORAGENAME"; then
            #mvlogger "$PRIMARYSTORAGENAME is zfs."
            POOLPCTUSED=$(zpool get -o value capacity $PRIMARYSTORAGENAME | tail -n 1 | tr -d '%')
            POOLBYTEUSED=$(zfs list -Hpo used $PRIMARYSTORAGENAME )
        else
            POOLPCTUSED=$(df --output=pcent "/mnt/$PRIMARYSTORAGENAME" | tail -n 1 | tr -d '%')
            POOLBYTEUSED=$(df --output=used --block-size=1 "/mnt/$PRIMARYSTORAGENAME" | tail -n 1)
        fi

        #Get the size threshold in bytes
        if is_zfs_mountpoint_fstype "/mnt/$PRIMARYSTORAGENAME"; then
            POOLSIZE=$(zfs list -Hp -o available,used $PRIMARYSTORAGENAME | awk '{print $1 + $2}')
        else
            POOLSIZE=$(df --output=size --block-size=1 "/mnt/$PRIMARYSTORAGENAME" | tail -n 1)
        fi
        PRIMARYSIZETHRESH=$(( $POOLSIZE * $FREEINGPCTLIMIT / 100))
        MOVESIZETHRESH=$(( $POOLSIZE * $MOVINGPCTTHRESHOLD / 100))

        # Print pool info:
        mvlogger "Primary storage: $PRIMARYSTORAGENAME - size: $(bytes_to_human $POOLSIZE) - used: $POOLPCTUSED % ($(bytes_to_human $POOLBYTEUSED))"
        mvlogger "Secondary storage: $SECONDARYSTORAGENAME"


        # Count number of storage pools on system for this share:
        if [ $PRIMARYSTORAGENAME = "user0" ] || [ $SECONDARYSTORAGENAME = "user0" ] ; then
            UNATTENDEDSTORAGE=$(find /mnt -type d -name "$SHARENAME" \
                            -not -path "/mnt/$PRIMARYSTORAGENAME/*" \
                            -not -path "/mnt/$SECONDARYSTORAGENAME/*" \
                            -not -path "/mnt/user*" \
                            -not -path "/mnt/disk*" \
                            -maxdepth 2 | cut -d'/' -f3)
        else
            UNATTENDEDSTORAGE=$(find /mnt -type d -name "$SHARENAME" \
                            -not -path "/mnt/$PRIMARYSTORAGENAME/*" \
                            -not -path "/mnt/$SECONDARYSTORAGENAME/*" \
                            -not -path "/mnt/user*" \
                            -maxdepth 2 | cut -d'/' -f3)
        fi

        # Determine sharepath
        SHAREPATH="/mnt/$PRIMARYSTORAGENAME/$SHARENAME"

        if [ ! -d "$SHAREPATH" ]; then  # && [ $SHAREUSECACHE != "no" ]
            # Do not process this pool if path does not exist
            mvlogger "$SHAREPATH does not exist. Is the share still used? Consider removing $SHARECFG if not."
            mvlogger "=> Skipping"
            continue  # Move to the next iteration of the loop
        fi

        if  [ $SHAREUSECACHE = "prefer" ] || ([ $REBALANCESHARES = "yes" ] && [ ! -z $UNATTENDEDSTORAGE ]); then
            SHAREPATH="/mnt/*/$SHARENAME"
        fi
        mvlogger "Share Information: Name: $SHARENAME - Path: $SHAREPATH"

        # Give information according to cache mode and thresholds
        if [ "$SHAREUSECACHE" = "prefer" ]; then
            mvlogger "Moving threshold: $MOVINGPCTTHRESHOLD% ($(bytes_to_human $MOVESIZETHRESH)) ; Freeing threshold: $FREEINGPCTLIMIT% ($(bytes_to_human $PRIMARYSIZETHRESH))"
            if [ $REBALANCESHARES = "yes" ] && [ ! -z $UNATTENDEDSTORAGE ]; then
                mvlogger "Mover action: $SECONDARYSTORAGENAME->$PRIMARYSTORAGENAME (cache:prefer). Rebalance shares option is selected"
                mvlogger "=> Files detected in $UNATTENDEDSTORAGE instead of $PRIMARYSTORAGENAME or $SECONDARYSTORAGENAME. They will be rebalanced back to these."
            elif [ $POOLPCTUSED -lt $FREEINGPCTLIMIT ]; then
                mvlogger "Mover action: $SECONDARYSTORAGENAME->$PRIMARYSTORAGENAME (cache:prefer). Pool is below priming threshold percentage: $POOLPCTUSED% < $FREEINGPCTLIMIT%."
                mvlogger "=> Will smart move newest files from $SECONDARYSTORAGENAME to $PRIMARYSTORAGENAME until threshold. Older files will be moved from $PRIMARYSTORAGENAME to $SECONDARYSTORAGENAME."
            elif [ $POOLPCTUSED -gt $MOVINGPCTTHRESHOLD ]; then
                mvlogger "Mover action: $SECONDARYSTORAGENAME->$PRIMARYSTORAGENAME (cache:prefer). Pool is above moving threshold percentage: $POOLPCTUSED% >= $MOVINGPCTTHRESHOLD%."
                mvlogger "=> Will smart move newest files from $SECONDARYSTORAGENAME to $PRIMARYSTORAGENAME until freeing threshold. Older files will be moved from $PRIMARYSTORAGENAME to $SECONDARYSTORAGENAME."
            else
                mvlogger "Mover action: $SECONDARYSTORAGENAME->$PRIMARYSTORAGENAME (cache:prefer). Pool is above freeing threshold percentage and below priming threshold percentage: $FREEINGPCTLIMIT% < $POOLPCTUSED% < $MOVINGPCTTHRESHOLD%."
                mvlogger "=> Skipping"
                continue
            fi
        elif [ $SHAREUSECACHE = "yes" ]; then
            mvlogger "Moving threshold: $MOVINGPCTTHRESHOLD% ($(bytes_to_human $MOVESIZETHRESH)) ; Freeing threshold: $FREEINGPCTLIMIT% ($(bytes_to_human $PRIMARYSIZETHRESH))"
            if [ $REBALANCESHARES = "yes" ]  && [ ! -z $UNATTENDEDSTORAGE ]; then
                mvlogger "Mover action: $PRIMARYSTORAGENAME->$SECONDARYSTORAGENAME (cache:yes). Rebalance shares option is selected"
                mvlogger "=> Files detected in $UNATTENDEDSTORAGE instead of $PRIMARYSTORAGENAME or $SECONDARYSTORAGENAME. They will be rebalanced to $SECONDARYSTORAGENAME."
            elif [ "$OMOVERTHRESH" != "" ] && [ $POOLPCTUSED -gt $OMOVERTHRESH ]; then
                mvlogger "Mover action: $PRIMARYSTORAGENAME->$SECONDARYSTORAGENAME (cache:yes). Move All from Primary->Secondary shares option is selected and pool is above move all threshold percentage:  $POOLPCTUSED% > $OMOVERTHRESH%."
                mvlogger "=> Moving all files from $PRIMARYSTORAGENAME to $SECONDARYSTORAGENAME"
            elif [ $POOLPCTUSED -lt $MOVINGPCTTHRESHOLD ]; then
                mvlogger "Mover action: $PRIMARYSTORAGENAME->$SECONDARYSTORAGENAME (cache:yes). Pool is below moving threshold percentage: $POOLPCTUSED% < $MOVINGPCTTHRESHOLD%."
                if [ $SYNCHRONIZECACHE = "yes" ] || [ $RESYNCHRONIZECACHE = "yes" ]; then
                    mvlogger "=> No file will be moved but share processed as Synchronize Primary->Secondary option is set"
                else
                    mvlogger "=> Skipping"
                    continue
                fi
            else
                mvlogger "Mover action: $PRIMARYSTORAGENAME->$SECONDARYSTORAGENAME (cache:yes). Pool is above moving threshold percentage:  $POOLPCTUSED% >= $MOVINGPCTTHRESHOLD%."
                mvlogger "=> Will smart move old files from $PRIMARYSTORAGENAME to $SECONDARYSTORAGENAME. Nothing will be moved from $SECONDARYSTORAGENAME to $PRIMARYSTORAGENAME"
            fi
        elif [ $SHAREUSECACHE = "only" ]; then
            if [ $REBALANCESHARES = "yes" ] && [ ! -z $UNATTENDEDSTORAGE ]; then
                mvlogger "Mover action: no action, only $PRIMARYSTORAGENAME used (cache:only). Rebalance shares option is selected"
                mvlogger "=> Files detected in $UNATTENDEDSTORAGE instead of $PRIMARYSTORAGENAME. They will be rebalanced to $PRIMARYSTORAGENAME"
            else
                mvlogger "Mover action: no action, only $PRIMARYSTORAGENAME used (cache:only)."
                mvlogger "=> Nothing will be moved. Share usage is taken into account in the calculation of the threshold for other shares." 
                if is_zfs_mountpoint_fstype "/mnt/$PRIMARYSTORAGENAME"; then
                    CACHESHARERESERVEDSPACE=$(zfs list -Hpo used $PRIMARYSTORAGENAME/$SHARENAME )
                else
                    mvlogger "Calculating share usage... (can take a moment)"
                    CACHESHARERESERVEDSPACE=$(du -sh "$SHAREPATH" | awk '{print $1}' | human_to_bytes)
                fi
                echo "$PRIMARYSTORAGENAME|$PRIMARYSTORAGENAME|$SHARENAME|only|9999999999|$PRIMARYSIZETHRESH|$CACHESHARERESERVEDSPACE|0|0|0|${SHAREPATH}" >>$FILTERED_FILELIST
                mvlogger "$PRIMARYSTORAGENAME/$SHARENAME used: $(bytes_to_human $CACHESHARERESERVEDSPACE)"
                continue
            fi
        elif [ $SHAREUSECACHE = "no" ]; then
            if [ $REBALANCESHARES = "yes" ] && [ ! -z $UNATTENDEDSTORAGE ]; then
                mvlogger "Mover action: no action, only $PRIMARYSTORAGENAME used (cache:no). Rebalance shares option is selected"
                mvlogger "=> Files detected in $UNATTENDEDSTORAGE instead of $PRIMARYSTORAGENAME. They will be rebalanced to $PRIMARYSTORAGENAME"
            else
                mvlogger "Mover action: no action, only $PRIMARYSTORAGENAME used (cache:no)."
                mvlogger "=> Skipping"
                continue
            fi
        fi
        
        # Do not process this pool if Share path is /mnt/cache
        if [ "$SHAREPATH" = "/mnt/cache" ]; then
            mvlogger "Skipping processing for /mnt/cache"
            continue  # Move to the next iteration of the loop
        fi

        #Base Find String
        # Alterate SHAREPATH on Cache=prefer or RebalanceShare=true
        if [ "$SHAREUSECACHE" = "prefer" ] || ( [ $REBALANCESHARES = "yes" ] && [ ! -z $UNATTENDEDSTORAGE ] ); then
            #"cache=prefer" pool, we replace path to list files in /mnt/user and process later
            FINDSTR="find $SHAREPATH ! -path \"/mnt/user*\" -type f -depth "
        else
            FINDSTR="find $SHAREPATH -type f -depth "
        fi

        #Addition of filters are conditionned by [ -z "$OMOVERTHRESH" ] || [ $POOLPCTUSED -le $OMOVERTHRESH ] to prevent filtering move all Primary->Secondary share
        #Add Age variables to find string
        if ([ -z "$OMOVERTHRESH" ] || [ $POOLPCTUSED -le $OMOVERTHRESH ]) && [ "$AGE" -ge 1 ]; then
            mvlogger "Adding Age"
            RAGE=$((AGE - 1))
            # Use Creation time or modification time according to settings
            if [ "$CTIME" == "yes" ]; then
                mvlogger "Age (creation time) ${AGE}"
                FINDSTR+=" -ctime +$RAGE"
            else
                mvlogger "Age (modification time) ${AGE}"
                FINDSTR+=" -mtime +$RAGE"
            fi
        fi

        #Add Size to find string
        if ([ -z "$OMOVERTHRESH" ] || [ $POOLPCTUSED -le $OMOVERTHRESH ]) && [ "$SIZE" -gt 0 ]; then
            mvlogger "Adding Size: ${SIZE}"
            FINDSTR+=" -size +${SIZE}M"
        fi


        #Add Ignore Hidden Files to find string
        if ([ -z "$OMOVERTHRESH" ] || [ $POOLPCTUSED -le $OMOVERTHRESH ]) && ([ "$SHAREUSECACHE" = "prefer" ] || [ "$SHAREUSECACHE" = "yes" ]) && [ "$IGNOREHIDDENFILES" == "yes" ]; then
            mvlogger "Skipping Hidden Files. File size are taken into account in the calculation of the threshold"
            FINDSTR+=" -not -path '*/\.*'"
            HIDDENFILES_RESERVED_SPACE=$(find "/mnt/$PRIMARYSTORAGENAME/$SHARENAME" -type f -path '*/\.*' -print0 | xargs -0 du -sh | awk '{ sum += $1 } END { print sum }')
            echo "$PRIMARYSTORAGENAME|$PRIMARYSTORAGENAME|$SHARENAME|skipped|9999999999|$PRIMARYSIZETHRESH|$SKIPPED_PATH_RESERVED_SPACE|0|0|0|/mnt/$PRIMARYSTORAGENAME/$SHARENAME hidden files" >>$FILTERED_FILELIST
            mvlogger "    Hidden files are using $(bytes_to_human $SKIPPED_PATH_RESERVED_SPACE)"
        fi
        #Add skipped files and folders
        FINDSTR+=" -not -name \".placeholder\"" #do not move placeholders
        # Add files from ignore list
        if ([ -z "$OMOVERTHRESH" ] || [ $POOLPCTUSED -le $OMOVERTHRESH ]) && ([ "$SHAREUSECACHE" = "prefer" ] || [ "$SHAREUSECACHE" = "yes" ] ) && [ -f "$SKIPFILESLIST" ] && grep -q "/mnt/$PRIMARYSTORAGENAME/$SHARENAME" "$SKIPFILESLIST"; then
            mvlogger "Skipping Files from List. File size are taken into account in the calculation of the threshold"
            mvlogger "List Path: ${SKIPFILESLIST}:"
            TOTAL_SKIPPED_PATH_RESERVED_SPACE=0
            while IFS= read -r skipped_path; do
                SKIPPED_PATH_RESERVED_SPACE=""
                skipped_path="${skipped_path%/}"  # Remove potential trailing slash
                if [ -d "$skipped_path" ]; then
                    # Size calculation
                    SKIPPED_PATH_RESERVED_SPACE=$(du -sh "${skipped_path}" | awk '{print $1}' | human_to_bytes)
                    # Add /* at the end for better reading in Mover_action file
                    skipped_path="${skipped_path}/*" # add slash asterisk
                    # Add filter to find string
                    FINDSTR+=" -not -path \"${skipped_path}\"" 
                elif [ -f "$skipped_path" ]; then
                    SKIPPED_PATH_RESERVED_SPACE=$(du -sh "${skipped_path}"  | awk '{print $1}'| human_to_bytes)
                    # Add filter to find string
                    FINDSTR+=" -not -name \"${skipped_path}\""
                fi
                if [ ! -z "$SKIPPED_PATH_RESERVED_SPACE" ]; then
                        echo "$PRIMARYSTORAGENAME|$PRIMARYSTORAGENAME|$SHARENAME|skipped|9999999999|$PRIMARYSIZETHRESH|$SKIPPED_PATH_RESERVED_SPACE|0|0|0|${skipped_path}" >> $FILTERED_FILELIST
                        TOTAL_SKIPPED_PATH_RESERVED_SPACE=$((TOTAL_SKIPPED_PATH_RESERVED_SPACE + SKIPPED_PATH_RESERVED_SPACE))
                    else
                        mvlogger "Error calculating size for $skipped_path"
                fi
            done <<< $(grep "/mnt/$PRIMARYSTORAGENAME/$SHARENAME" "$SKIPFILESLIST")
            mvlogger "    Ignored files are using $(bytes_to_human $TOTAL_SKIPPED_PATH_RESERVED_SPACE)"
        fi

        #Add Skipfilelist to find string
        TOTAL_SKIPPED_FILETYPES_RESERVED_SPACE=0
        if ([ -z "$OMOVERTHRESH" ] || [ $POOLPCTUSED -le $OMOVERTHRESH ]) && ([ "$SHAREUSECACHE" = "prefer" ] || [ "$SHAREUSECACHE" = "yes" ] ) && ([ ! -z "$SKIPFILETYPES" ] || [ ! -z "$globalSkipFileTypes" ]); then
            mvlogger "Skipping Filetypes from List. File sizes are taken into account for the calculation of the threshold"
            for ext in $([ $overrideFlag = 1 ] && echo -n "$SKIPFILETYPES") $(echo -n "$globalSkipFileTypes"); do
                FINDSTR+=" -not -name \*.$ext"
                SKIPPED_FILETYPES_RESERVED_SPACE=$(find /mnt/$PRIMARYSTORAGENAME/$SHARENAME -type f -name \*.$ext -print0 | xargs -0 du -sh | awk '{print $1}' | human_to_bytes | awk '{ sum += $1 } END { print sum }')
                if [ $? -eq 0 ]; then
                    echo "$PRIMARYSTORAGENAME|$PRIMARYSTORAGENAME|$SHARENAME|skipped|9999999999|$PRIMARYSIZETHRESH|$SKIPPED_FILETYPES_RESERVED_SPACE|0|0|0|$SHAREPATH/**/*.$ext" >>$FILTERED_FILELIST
                    TOTAL_SKIPPED_FILETYPES_RESERVED_SPACE=$(( TOTAL_SKIPPED_FILETYPES_RESERVED_SPACE + SKIPPED_FILETYPES_RESERVED_SPACE))
                else
                    mvlogger "Error calculating size for $skipFileTypes"
                fi
            done
            mvlogger "    Ignored filetypes are using $(bytes_to_human $TOTAL_SKIPPED_FILETYPES_RESERVED_SPACE)"
        fi

        #Add Size to find string and pass to awk to modify some fields
        FINDSTR+=" -printf '%T@|%s|%S|%n|%i|%p|\0' | awk -v RS='\0' -v FS='|' -v SPARSENESS=\$SPARSENESS -v PRIMARYSTORAGENAME=\$PRIMARYSTORAGENAME -v SECONDARYSTORAGENAME=\$SECONDARYSTORAGENAME -v SHARENAME=\$SHARENAME -v SHAREUSECACHE=\$SHAREUSECACHE -v PRIMARYSIZETHRESH=\$PRIMARYSIZETHRESH"
        if ([ -z "$OMOVERTHRESH" ] || [ $POOLPCTUSED -le $OMOVERTHRESH ]) && [ "$SPARSENESS" -gt 0 ]; then
            mvlogger "Adding Sparseness size ${SPARSENESS}"
            FINDSTR+="' \$3 > 0.SPARSENESS ' {"
        else
            FINDSTR+=" ' {"
        fi
        # Get Birthtime with stat if CTIME option enabled
        if [ $CTIME == "yes" ]; then
            FINDSTR+="cmd=\"stat -c%W \\\"\"\$6\"\\\"\";
                cmd | getline result;
                close(cmd);
                sub(/\\n\$/, \"\", result);
                \$1 = result;
            "
        fi
        # Get size with du if on zfs mountpoint with compression enabled
#        if is_zfs_mountpoint_fstype "/mnt/$PRIMARYSTORAGENAME" && [ "$(zfs get compression -Hpo value "$PRIMARYSTORAGENAME/$SHARENAME")" == "on" ]; then
#            FINDSTR+="cmd = \"du -sh \\\"\"\$6\"\\\" | numfmt --from=iec\";
#                cmd | getline result;
#                close(cmd);
#                sub(/\\n\$/, \"\", result);
#                \$2 = result;
#            " 
#        fi
        FINDSTR+="printf \"%s|%s|%s|%s|%d|%d|%d|%d|%d|%d|%s\\n\", PRIMARYSTORAGENAME, SECONDARYSTORAGENAME, SHARENAME, SHAREUSECACHE, \$1, PRIMARYSIZETHRESH, \$2, \$3, \$4, \$5, \$6}'"

        if [ -d "$SHAREPATH" ] || [ "$SHAREUSECACHE" = "prefer" ] || [ $REBALANCESHARES = "yes" ] ; then
            [ $CTIME == "yes" ] && mvlogger "Filtering $SHARENAME files... (can take a moment)"
            eval "$FINDSTR>>$FILTERED_FILELIST"
            mvlogger "Updated Filtered filelist: $FILTERED_FILELIST for $SHARENAME"
        fi
    done

    # Sort the file list by PRIMARYSTORAGENAME, shareusecache, creation/modification time, and inode, keeping /mnt/disk at the end
    sort -t '|' -k1,1 -k4,4 -k5,5nr -k10,10 -o "$FILTERED_FILELIST" "$FILTERED_FILELIST"
    # Need to remove duplicate (cache and array) 
    if grep -q "/mnt/disk" $FILTERED_FILELIST; then
        mvlogger "$(titleLine 'Optimizing Filtered Filelist' '-')"
        tempfile1=$(mktemp)
        tempfile2=$(mktemp)

        mvlogger "Removing duplicate giving priority to cache"
        grep -v "/mnt/disk" $FILTERED_FILELIST > "$tempfile1"
        grep "/mnt/disk" $FILTERED_FILELIST > "$tempfile2"
        cat "$tempfile1" "$tempfile2" | awk -F"|" '
            function is_duplicate(key) {
                if (prev[key]) {
                    return true
                } else {
                    prev[key] = 1
                    return false
                }
            }
            function extract_path(filepath) {
                # Optimized path extraction using sub
                sub(/^\/[^\/]+\/[^\/]+\/[^\/]+\//, "", filepath)
                return filepath
            }
            BEGIN {
                FS = OFS = "|"

                if (system("[ -e \"" SOFTSTOPFILE "\" ]") == 0) {
                    exit 1
                }
                # Initialize previous inode and dictionary for pool sums
                previous_inode = 0

                # Print new header
                printf "PRIMARYSTORAGENAME|SECONDARYSTORAGENAME|SHARENAME|SHAREUSECACHE|MODIFICATIONTIME|PRIMARYSIZETRESH|FILESIZE|SPARSENESS|NBLINKS|INODE|FILEPATH\n"
            }

            NR > 1 { # Main loop, skip header line
                if (system("[ -e \"" SOFTSTOPFILE "\" ]") == 0) {
                    exit 1
                }
                file_path = extract_path($11)

                # Handle duplicates
                key = $3 "|" $5 "|" $7 "|" file_path
                if (! is_duplicate(key)) {
                    printf "%s|%s|%s|%s|%d|%d|%d|%d|%d|%d|%s\n", $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11
                }
            }' > "$FILTERED_FILELIST"
        mvlogger "Sorting"
        sort -t '|' -k1,1 -k4,4 -k5,5nr -k10,10 -o "$FILTERED_FILELIST" "$FILTERED_FILELIST"
        rm $tempfile1 $tempfile2
    fi
}

decideMoveActions () {
    if [ -e $SOFTSTOPFILE ]; then
        [ $SOFTSTOP_ACK = true ] || mvlogger "Soft Stopping requested" && SOFTSTOP_ACK=true
        return
    fi

    # Title line
    mvlogger "$(titleLine 'ANALYSING MOVING ACTIONS' '*')"

    #If overrideFLag set restore settings
    if [ $overrideFlag = 1 ]; then
        overrideFlag=0
        #Get Mover Tuning Settings
        mvlogger "Restore global settings"
        mvlogger "-----------------------"
        getMoverSettings $MOVERTUNINGCFGFILE
        mvlogger "------------------------------------------------------------------------"
    fi

    ## Loop Filtered Filelist and for each file decide to move to/from the array
    mvlogger "Deciding the action (move/sync/keep) for each file. There are $(grep "/mnt" $FILTERED_FILELIST | wc -l) files, it can take a while..."

    ## Proceed with files
    # Fields: $1:PRIMARYSTORAGENAME $2:SECONDARYSTORAGENAME $3:SHARENAME $4:SHAREUSECACHE $5:MODIFICATIONTIME
    # $6:PRIMARYSIZETRESH $7:FILESIZE $8:SPARSENESS $9:NBLINKS $10:INODE $11:FILEPATH
    
    awk -F"|" -v AGE="$AGE" -v STATS_FILE="$STATS_FILE" -v LASTCACHESYNC=$LASTCACHESYNC -v SOFTSTOPFILE="$SOFTSTOPFILE" -v SYNCHRONIZECACHE=$SYNCHRONIZECACHE -v REBALANCESHARES=$REBALANCESHARES -v RESYNCHRONIZECACHE=$RESYNCHRONIZECACHE '
        BEGIN {
            FS = OFS = "|"

            if (system("[ -e \"" SOFTSTOPFILE "\" ]") == 0) {
                exit 1
            }
            # Initialize previous inode and dictionary for pool sums
            previous_inode = 0

            # Print new header
            printf "PRIMARYSTORAGENAME|SHARENAME|SHAREUSECACHE|MODIFICATIONTIME|FILESIZE|POOLSUM|THRESHOLD|NBLINKS|INODE|ACTION|FROM|TO|FILEPATH\n"
        }

        NR > 1 { # Main loop, skip header line
            if (system("[ -e \"" SOFTSTOPFILE "\" ]") == 0) {
                exit 1
            }

            # Extract the part after the first three slashes (/mnt/pool/) in the eleventh field (filepath)
            split($11, path_parts, "/")
            storage_path = path_parts[1] "/" path_parts[2] "/" path_parts[3]
            storage_pool = path_parts[3]
            file_path = ""
            for (i = 4; i <= length(path_parts); i++) {
                file_path = file_path path_parts[i]
                if (i < length(path_parts)) {
                    file_path = file_path "/"
                }
            }

            # Handle hardlinks
            if ($9 > 1) {
                key = $1 "|" $2 "|" $3 "|" $10
                if (prev[key]) {
                    $7 = 0
                } else {
                    prev[key] = 1
                }
            }


            # Update pool sum (use pool_sums[$1])
            pool_sums[$1] += $7

            # Take decision
            if (storage_pool == $1) {
                # Make decision to move or keep:
                action = (($4 != "only" && $4 != "skipped") && (($4 == "prefer" || $4 == "yes") && (pool_sums[$1] >= $6 && AGE == -1) || AGE >= 0) || $4 == "no" ) ? "move from primary" : "keep on primary"

                if (action == "keep on primary")  {
                    source = storage_path
                    if (SYNCHRONIZECACHE == "yes" && ($4 != "only" && $4 != "skipped") && ( $5 >= LASTCACHESYNC || RESYNCHRONIZECACHE == "yes" ) )  {
                        action = "sync from primary"
                        destination = "/mnt/" $2
                        pool_stats[$1]["files_from_primary"] += 1
                        pool_stats[$1]["total_size_from_primary"] += $7
                    } else {
                        destination = "none"
                    }
                }

                # Update pool statistics and set destination if moving the file
                if (action == "move from primary") {
                    source = storage_path
                    destination = "/mnt/" $2
                    pool_stats[$1]["files_from_primary"] += 1
                    pool_stats[$1]["total_size_from_primary"] += $7
                }
            } else {
                if (storage_pool == "user" || ($2 == "user0" && storage_pool ~ "disk")) {
                    # Make decision to move or keep:
                    action = (($4 == "only" && REBALANCESHARES == "yes") || ($4 == "prefer" && ((pool_sums[$1] <= $6 && AGE == -1) || AGE >= 0))) ? "move from secondary" : "keep on secondary"

                    if (action == "keep on secondary") {
                        source = storage_path
                        destination = "none"
                    }

                    # Update pool statistics and set destination if moving the file
                    if (action == "move from secondary") {
                        source = storage_path
                        destination = "/mnt/" $1
                        if (SYNCHRONIZECACHE == "yes" && ($4 != "only" && $4 != "skipped"))  {
                            action = "sync from secondary"
                        }
                        pool_stats[$1]["files_from_secondary"] += 1
                        pool_stats[$1]["total_size_from_secondary"] += $7
                    }
                } else {
                    if (REBALANCESHARES == "yes") {
                        action = ($4 == "only" || ($4 == "prefer" && pool_sums[$1] <= $6)) ? "move unattended to primary" : "move unattended to secondary"
                        if (action == "move unattended to primary" && SYNCHRONIZECACHE == "yes" && $4 != "only" )  {
                            source = storage_path
                            destination = "/mnt/" $1
                            pool_stats[$1]["files_from_unattended"] += 1
                            pool_stats[$1]["total_size_from_unattended"] += $7
                            # Add a line in mover_action as intermediary step
                            printf "%s|%s|%s|%d|%d|%d|%d|%d|%s|%s|%s|%s|%s\n", $1, $3, $4, $5, $7, pool_sums[$1], $6, $9, $10, action, source, destination, file_path
                            action="sync unattended to secondary"
                            source = "/mnt/" $1
                            destination = "/mnt/" $2
                        } else {
                            if (action == "move unattended to primary") {
                                source = storage_path
                                destination = "/mnt/" $1
                            } else {
                                action = "move unattended to secondary"
                                source = storage_path
                                destination = "/mnt/" $2
                            }
                        }
                        pool_stats[$1]["files_from_unattended"] += 1
                        pool_stats[$1]["total_size_from_unattended"] += $7
                    } else {
                        action="keep misplaced" #If not on primary nor secondary, file will be tagged misplaced                
                        source = storage_path
                        destination = "none"
                    }
                }
            }

            # Create action list
            printf "%s|%s|%s|%d|%d|%d|%d|%d|%s|%s|%s|%s|%s\n", $1, $3, $4, $5, $7, pool_sums[$1], $6, $9, $10, action, source, destination, file_path
        }
        END {
            if (system("[ -e \"" SOFTSTOPFILE "\" ]") == 0) {
                exit 1
            }

            # Write statistics header line
            printf "PRIMARYSTORAGENAME|FILES_FROM_PRIMARY|SIZE_FROM_PRIMARY|FILES_FROM_SECONDARY|SIZE_FROM_SECONDARY\n" > "'"${STATS_FILE}"'"

            # Loop through pool_stats and write data to stats_file
            total_files_from_primary = 0
            total_size_from_primary = 0
            total_files_from_secondary = 0
            total_size_from_secondary = 0
            total_files_from_unattended = 0
            total_size_from_unattended = 0

            for (poolname in pool_stats) {
                files_from_primary = pool_stats[poolname]["files_from_primary"]
                size_from_primary = pool_stats[poolname]["total_size_from_primary"]
                files_from_secondary = pool_stats[poolname]["files_from_secondary"]
                size_from_secondary = pool_stats[poolname]["total_size_from_secondary"]
                files_from_unattended = pool_stats[poolname]["files_from_unattended"]
                size_from_unattended = pool_stats[poolname]["total_size_from_unattended"]

                total_files_from_primary += files_from_primary
                total_files_from_secondary += files_from_secondary
                total_size_from_primary += size_from_primary
                total_size_from_secondary += size_from_secondary
                total_files_from_unattended += files_from_unattended
                total_size_from_unattended += size_from_unattended
                printf("%s|%d|%d|%d|%d|%d|%d\n", poolname, files_from_primary, size_from_primary, files_from_secondary, size_from_secondary, files_from_unattended, size_from_unattended) >> "'"${STATS_FILE}"'"
            }

            # Print overall totals to stats file
            printf("TOTAL|%d|%d|%d|%d|%d|%d\n", total_files_from_primary, total_size_from_primary, total_files_from_secondary, size_from_secondary, total_files_from_unattended, size_from_unattended) >> "'"${STATS_FILE}"'"

    }' "$FILTERED_FILELIST" > $MOVER_ACTIONLIST
    if [ -e $SOFTSTOPFILE ]; then
        [ $SOFTSTOP_ACK = true ] || mvlogger "Soft Stopping requested" && SOFTSTOP_ACK=true
        return
    fi

    ## Printing some stats
    TOTALPRIMARYFILES=$(grep "TOTAL" "$STATS_FILE" | cut -d "|" -f 2)
    TOTALPRIMARYSIZE=$(grep "TOTAL" "$STATS_FILE" | cut -d "|" -f 3)
    TOTALSECONDARYFILES=$(grep "TOTAL" "$STATS_FILE" | cut -d "|" -f 4)
    TOTALSECONDARYSIZE=$(grep "TOTAL" "$STATS_FILE" | cut -d "|" -f 5)
    TOTALUNATTENDEDFILES=$(grep "TOTAL" "$STATS_FILE" | cut -d "|" -f 6)
    TOTALUNATTENDEDSIZE=$(grep "TOTAL" "$STATS_FILE" | cut -d "|" -f 7)

    # Add total files and size
    TOTALFILES=$((TOTALPRIMARYFILES + TOTALSECONDARYFILES + TOTALUNATTENDEDFILES))
    TOTALSIZE=$((TOTALPRIMARYSIZE + TOTALSECONDARYSIZE + TOTALUNATTENDEDSIZE))
    UNATTENDEDFILES=$(grep "keep misplaced" $MOVER_ACTIONLIST |wc -l)

    # Print stats
    if [ $UNATTENDEDFILES -gt 0 ]; then
        mvlogger "Warning: $UNATTENDEDFILES files are stored outside their primary or secondary storages and won't be moved according to actual settings."
    fi
    if [ $TOTALFILES -gt 0 ]; then
        mvlogger "A total of $TOTALFILES files representing $(bytes_to_human $TOTALSIZE) will be moved/synced:"
        if [ $TOTALPRIMARYFILES -gt 0 ]; then
            # Read data line by line using while loop
            while IFS="|" read -r PRIMARYSTORAGENAME PRIMARYFILES PRIMARYSIZE SECONDARYFILES SECONDARYSIZE; do
                if [ -e $SOFTSTOPFILE ]; then
                    [ $SOFTSTOP_ACK = true ] || mvlogger "Soft Stopping requested" && SOFTSTOP_ACK=true
                    break
                fi
                # Check if line is not the total line nor the header line
                if [ $PRIMARYSTORAGENAME != "TOTAL" ] && [ $PRIMARYSTORAGENAME != "PRIMARYSTORAGENAME" ]; then
                    mvlogger "- $PRIMARYFILES files representing $(bytes_to_human $PRIMARYSIZE) will be moved/sync from $PRIMARYSTORAGENAME to secondary"
                fi
            done < "$STATS_FILE"
        fi
        if [ $TOTALSECONDARYFILES -gt 0 ]; then
            mvlogger "- $TOTALSECONDARYFILES new files representing $(bytes_to_human $TOTALSECONDARYSIZE) will then be moved/synced from secondary to primary"
        fi
        if [ $TOTALUNATTENDEDFILES -gt 0 ]; then
            mvlogger "- $TOTALUNATTENDEDFILES files from unattended storage and representing $(bytes_to_human $TOTALUNATTENDEDSIZE) will then be rebalanced from unattended shares to primary and/or secondary"
        fi
    else
        mvlogger "No new files will be moved/synced from primary to secondary"
        mvlogger "No new files will be moved/synced from secondary to primary"
    fi
}

# Internal move engine
processTheMoves () {
    if [ -e $SOFTSTOPFILE ]; then
        [ $SOFTSTOP_ACK = true ] || mvlogger "Soft Stopping requested" && SOFTSTOP_ACK=true
        return
    fi

    mvlogger "$(titleLine 'LET THE MOVING SHOW BEGIN !' '*')"
    if [ $ENABLETURBO = "yes" ] ; then
        mvlogger "Forcing turbo write on"
        [ $TESTMODE != "yes" ] && /usr/local/sbin/mdcmd set md_write_method 1
    fi
    # MOVER_ACTIONLIST header: 
    # PRIMARYSTORAGENAME|SHARENAME|SHAREUSECACHE|MODIFICATIONTIME|FILESIZE|POOLSUM|THRESHOLD|NBLINKS|INODE|ACTION|FROM|TO|FILEPATH

    #Initialise stats:
    REMAINING_ARRAY_SIZE=$TOTALSECONDARYSIZE
    REMAINING_ARRAY_FILES=$TOTALSECONDARYFILES
    REMAINING_CACHE_SIZE=$TOTALPRIMARYSIZE
    REMAINING_CACHE_FILES=$TOTALPRIMARYFILES


    # Dry mode info:
    if [ $TESTMODE = "yes" ]; then
        mvlogger "Test Mode: yes, running rsync in dry-mode for moving and syncing"
    fi

    # Loop through each line in the action list (except those where action is keep), starting by cache and then array
    GREP_CMDS=(
        "grep '|move from primary|' '$MOVER_ACTIONLIST'"
        "grep '|move unattended to secondary|' '$MOVER_ACTIONLIST'"
        "grep '|sync from primary|' '$MOVER_ACTIONLIST'"
        "grep '|move from secondary|' '$MOVER_ACTIONLIST'"
        "grep '|sync from secondary|' '$MOVER_ACTIONLIST'"
        "grep '|move unattended to primary|' '$MOVER_ACTIONLIST'"
        "grep '|sync unattended to secondary|' '$MOVER_ACTIONLIST'"
    )
    # Ignore "grep '|keep on primary|' '$MOVER_ACTIONLIST'" action because it's not an action

    for GREP_CMD in "${GREP_CMDS[@]}"; do
        while IFS="|" read -r PRIMARYSTORAGENAME SHARENAME SHAREUSECACHE MODIFICATIONTIME FILESIZE POOLSUM THRESHOLD NBLINKS INODE ACTION SOURCE DESTINATION FILEPATH; do
            if [ -e $SOFTSTOPFILE ]; then
                [ $SOFTSTOP_ACK = true ] || mvlogger "Soft Stopping requested" && SOFTSTOP_ACK=true
                return
            fi

            # Skip if ~blank~ line
            if [ -z "$NBLINKS" ] || [ -z "$FILESIZE" ] || [ -z "$ACTION" ] || [ -z "$FILEPATH" ]; then
                continue
            fi
            # Skip filesize = 0 (hardlinks already handled in a previous loop)
            NBLINKS="${NBLINKS:-0}"
            if [ "$NBLINKS" -gt 1 ] && [ "$FILESIZE" = 0 ] ; then
                continue
            fi
            #Prepare RSYNC_CMD
            RSYNC_CMD="rsync --archive --xattrs --relative --hard-links"

            if [ $TESTMODE = "yes" ]; then
                RSYNC_CMD+=" --dry-run"
            fi

            # Get files to rsync:
            SYNCED_FILES=$(grep "|$NBLINKS|$INODE|" $MOVER_ACTIONLIST | grep "|$MODIFICATIONTIME|" |  grep "ary|$SOURCE|" | cut -d'|' -f13 | while read line; do echo "\"$SOURCE/./$line\""; done | tr '\n' ' ')
            GUI_CURRENT_FILE=$SYNCED_FILES

            # Check if list is complete
            NBFILES=$(($(echo "$SYNCED_FILES" | grep -o '" "' | wc -l) + 1))
            if [ $NBFILES != $NBLINKS ]; then
                mvlogger "Expected $NBLINKS files, got $NBFILES. Not moving $SYNCED_FILES to prevent breaking hardlinks"
                continue
            fi

            # Proceed with rsync
            if [[ "$ACTION" =~ "move" ]]; then
                # Some verbosity
                GUI_ACTION="Moving to $DESTINATION/ $([ $NBLINKS -gt 1 ] && echo '(preserving hardlinks)')"
                mvlogger "Moving $SYNCED_FILES to  $DESTINATION/ $([ $NBLINKS -gt 1 ] && echo '(preserving hardlinks)')"
                # Finalize RSYNC_CMD
                RSYNC_CMD+=" --remove-source-files"   
            elif [[ "$ACTION" =~ "sync" ]] ; then 
                # Some verbosity
                GUI_ACTION="Synchronizing to $DESTINATION/ $([ $NBLINKS -gt 1 ] && echo '(preserving hardlinks)')"
                mvlogger "Synchronizing  $SYNCED_FILES to  $DESTINATION/ $([ $NBLINKS -gt 1 ] && echo '(preserving hardlinks)')"
            fi

            moverStatusWrite
            # Perform rsync. relative paths and hardlinks
            eval "$RSYNC_CMD" "$SYNCED_FILES" "$DESTINATION/"

            # Clean parent folder if empty
            if [[ "$ACTION" =~ "move" ]] && [ $CLEANFOLDERS = "yes" ]; then
                while IFS= read -r FILE; do
                    DIR=$(dirname "$FILE")
                    if [ -e "$DIR/.placeholder" ]; then
                        count=".placeholder"
                    else
                        count=$(ls -A "$DIR" | wc -l)
                    fi
                    if [ "$count" = ".placeholder" ] || [ "$count" != 0 ]; then
                        echo "Not deleting dir containing $count file$([ "$count" != ".placeholder" ] && echo "s"): ${DIR}"
                    else
                        echo "Deleting empty dir ${DIR}"
                        if [ $TESTMODE = "no" ]; then
                            rmdir "${DIR}"
                        fi
                    fi
                done <<< $(echo "$SYNCED_FILES" | sed 's|/./|/|g' | xargs -n1 echo)
            fi
            # Setting mover status
            if [[ "$SOURCE" =~ $PRIMARYSTORAGENAME ]]; then
                REMAINING_CACHE_SIZE=$(($REMAINING_CACHE_SIZE - $FILESIZE))
                REMAINING_CACHE_FILES=$(($REMAINING_CACHE_FILES - $NBLINKS))
                mvlogger "$REMAINING_CACHE_FILES files remaining from caches to array  $(bytes_to_human $REMAINING_CACHE_SIZE)"
            else
                REMAINING_ARRAY_SIZE=$(($REMAINING_ARRAY_SIZE - $FILESIZE))
                REMAINING_ARRAY_FILES=$(($REMAINING_ARRAY_FILES - $NBLINKS))
                mvlogger "$REMAINING_ARRAY_FILES files remaining from array to caches $(bytes_to_human $REMAINING_ARRAY_SIZE)"
            fi
            moverStatusWrite
        done <<< $(eval $GREP_CMD)
    done

    mvlogger "$(titleLine "Cleaning up" '-')"

    if [ $TESTMODE != "yes" ] && [ ! -e $SOFTSTOPFILE ] && grep -q "|sync from" $MOVER_ACTIONLIST ; then
        EPOCH=$(date --date="${NOW:0:4}-${NOW:5:2}-${NOW:8:2} ${NOW:11:2}:${NOW:13:2}:${NOW:15:2}" +%s)
        mvlogger "Set last sync date in config file: $EPOCH = $NOW"
        if grep -q "lastCacheSync=" "$MOVERTUNINGCFGFILE"; then
            sed -i "s/lastCacheSync=.*$/lastCacheSync=$EPOCH/" "$MOVERTUNINGCFGFILE"
        else
            echo "lastCacheSync=$EPOCH" >> "$MOVERTUNINGCFGFILE"
        fi
    fi

    if [ $ENABLETURBO = "yes" ] ; then
        turbo_write_mode=$(cat /var/local/emhttp/var.ini | grep "md_write_method=" | cut -d'"' -f2)
        mvlogger "Restoring original turbo write mode $( ( [ $turbo_write_mode = "auto" ] && echo "Auto (read/modify/write)" ) || ( [ $turbo_write_mode -eq 1 ] && echo "Turbo writes (Reconstruct)" ) || echo "Read/modify/write" )"
        [ $TESTMODE != "yes" ] && /usr/local/sbin/mdcmd set md_write_method $turbo_write_mode
    fi

}

start() {
    [ $LOGLEVEL = 0 ] && echo "Starting Mover tuning plugin.."
    mvlogger "$(titleLine "Mover Tuning Plugin version $(grep "version=" "$MOVERTUNINGCFGFILE" | cut -d '"' -f 2)" '*')"
    # Do not start if already running
    if [ -f $PIDFILE ] && ps h $(cat $PIDFILE) | grep mover; then
            mvlogger "mover: already running"
            exit 1
    fi
    echo $$ > $PIDFILE

    #Remove any old /var/run/moversoft.stop files
    if [ -e $SOFTSTOPFILE ]; then
        rm $SOFTSTOPFILE
        mvlogger "Mover soft stop file removed."
    fi

    # Only start if config OK
    if [ -f $UNRAIDCFGFILE ]; then
        if grep -qs 'shareMoverLogging="yes"' $UNRAIDCFGFILE; then
            mvlogger "Log Level: $LOGLEVEL"
        fi
    else 
        mvlogger "Fatal error: $UNRAIDCFGFILE does not exist, check this."
        rm $SOFTSTOPFILE
        exit 2
    fi
    if [ -f $MOVERTUNINGCFGFILE ]; then
        #Get Mover Tuning Settings
        mvlogger "$(titleLine "Global settings" '-')"
        getMoverSettings $MOVERTUNINGCFGFILE
    else
        mvlogger "Fatal error: $MOVERTUNINGCFGFILE does not exist, check this."
        rm $SOFTSTOPFILE
        exit 3
    fi

    # Only start if cache enabled and present
    if ! grep -qs 'shareCacheEnabled="yes"' $UNRAIDCFGFILE && [ $( ls /boot/config/pools/*.cfg | wc -l) -lt 2 ]; then
        mvlogger "Fatal error: cache not enabled and less than 2 pools set up."
        rm $SOFTSTOPFILE
        exit 4
    fi
    if ! mountpoint -q /mnt/user0 && [ $( ls /boot/config/pools/*.cfg | wc -l) -lt 2 ]; then
        mvlogger "Fatal error: cache not present, or only cache present, and less than 2 pools set up."
        rm $SOFTSTOPFILE
        exit 5
    fi

    #Delete helper files over 5 days old.
    find /tmp/ca.mover.tuning/* -daystart -mtime +5 -name '*.list' -delete

    #Delete Mover Log files over 10 days old.
    find /tmp/ca.mover.tuning/* -daystart -mtime +10 -name '*.log' -delete

    #Delete Mover List files with 0 lines.
    find /tmp/ca.mover.tuning/* -size 0 -name '*.list' -delete

    #Let find find hidden files.
    shopt -s nullglob dotglob

    #Run mover before script if specified.
    if [ ! -z "$BEFORESCRIPT" ] && [ ! -e $SOFTSTOPFILE ]; then
        if [ -f "$BEFORESCRIPT" ] && [ -x "$BEFORESCRIPT" ]; then
            mvlogger "Launching before script: $BEFORESCRIPT"
            eval \"$BEFORESCRIPT\"
            mvlogger "Before script finished"
        else
            mvlogger "Before script file $BEFORESCRIPT does not exist or is not executable"
            mvlogger "Skipping"
        fi
    fi

    # Proceed with filter and create filtered filelist and action filelist
    [ ! -e $SOFTSTOPFILE ] && createFilteredFilelist
    [ ! -e $SOFTSTOPFILE ] && decideMoveActions
    # And now.. we move! ... if needed
    [ ! -e $SOFTSTOPFILE ] &&  [ $TOTALFILES -gt 0 ] && processTheMoves

    #Write the final status just for sure
    moverStatusWrite
 
    #Run mover after script if specified.
    if [ ! -z "$AFTERSCRIPT" ] && [ ! -e $SOFTSTOPFILE ]; then
        if [ -f "$AFTERSCRIPT" ] && [ -x "$AFTERSCRIPT" ]; then
            mvlogger "Launching after script: $AFTERSCRIPT"
            eval \"$AFTERSCRIPT\"
            mvlogger "After script finished with result: $( [ $? = 0 ] && echo "OK" || echo "$?")"
        else
            mvlogger "After script file $AFTERSCRIPT does not exist or is not executable."
            mvlogger "Skipping"
        fi
    fi

    #Reset run-once settings
    if [ ! -e $SOFTSTOPFILE ]; then
        for config_file in $MOVERTUNINGCFGFILE /boot/config/plugins/ca.mover.tuning/shareOverrideConfig/*.cfg; do
            resetRunOnceMoverSettings $MOVERTUNINGCFGFILE
        done
    fi

    mvlogger "Cleaning lock and stop files"
    for file in $PIDFILE $SOFTSTOPFILE; do
        [[ -e "$file" ]] && rm -f "$file"
    done

    mvlogger "$(titleLine 'WE ARE DONE !' '*')"
    [ $LOGLEVEL = 0 ] && echo "Mover tuning plugin done!"
}

killtree() {
    local pid=$1 child

    for child in $(pgrep -P $pid); do
        killtree $child
    done
    [ $pid -ne $$ ] && kill -TERM $pid
}

# Caution: stopping mover like this can lead to partial files on the destination
# and possible incomplete hard link transfer.  Not recommended to do this.
stop() {
    if [ ! -f $PIDFILE ]; then
        echo "mover: not running"
        exit 0
    fi
    killtree $(cat $PIDFILE)
    sleep 2
    rm -f $PIDFILE
    echo "mover: stopped"
    exit
}

softstop() {
    echo "Soft Stop Requested"
    if [ ! -f $PIDFILE ]; then
        echo "mover: not running"
        exit 0
    fi
    touch $SOFTSTOPFILE
    exit
}

status() {
    if [ -f $PIDFILE ]; then
	echo "mover: running"
	cat $PIDFILE
	exit 0
    else
	echo "mover: not running"
    fi
    exit
}

case $1 in
start)
    start
    ;;
stop)
    stop
    ;;
softstop)
    softstop
    ;;
status)
    status
    ;;
*)
    echo "Usage: $0 (start|stop|softstop|status)"
    ;;
esac

# That's all folks :)

